﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using HtmlAgilityPack;
using System.IO;

namespace rftc
{
    internal class Crawler
    {
        private string pageLink;//用于存储需要抓取的页面的链接

        /// <summary>
        /// 初始化Crawler类
        /// </summary>
        /// <param name="PageLink">需要抓取的页面的链接</param>
        internal Crawler(string PageLink)
        {
            pageLink = PageLink;
        }


        /// <summary>
        /// 抓取页面并返回一个HtmlDocument实例
        /// </summary>
        /// <returns>抓取后生产的HtmlDocument实例</returns>
        internal HtmlDocument GetPage()
        {
            WebRequest request = HttpWebRequest.Create(pageLink);
            request.Method = "GET";
            Stream s = request.GetResponse().GetResponseStream();//获取响应流
            HtmlDocument document = new HtmlDocument();
            document.Load(s);//加载HTML文档
            s.Close();
            return document;
        }
    }
}
